package FocusCrawler::Accelerated;
our @ISA       = qw(FocusCrawler);
our $VERSION = 1.0;

use warnings;
use strict;

use FocusCrawler qw(&get_new_url &do_leaf_numbering_recursive &collect_features %params %dumps %digests %urls $frontier_db);
use Readonly;
use HTML::TreeBuilder;
use Storable;
use MLDBM qw(DB_File Storable);
use Lingua::EN::StopWords qw(%StopWords);
use Algorithm::NaiveBayes;
use BerkeleyDB;

our $nb;
my $window_width;

sub init {
    my $package = shift;
    $package->SUPER::init(@_);
    $window_width = $params{window_width};
    $nb = new Algorithm::NaiveBayes;
    if ($params{resume_flag} and not $params{learn_flag}) {
        $nb->{model} = Storable::retrieve($params{NB_file});
    }
}

sub reorder_frontier {
    my $package = shift;
    print STDERR "Start reordering Urls in frontier...\n\n" unless $params{silent};

    my $cursor = $frontier_db->db_cursor();
    my ( $priority, $url ) = ( '', '' );
    while ( $cursor->c_get( $priority, $url, DB_NEXT ) == 0 ) {
        delete $urls{$url};
        $cursor->c_del();
    }
    undef $cursor;

    my $instances_cnt = 0;
    tie my %meta_docs,     'MLDBM',   $params{metas_db_file}     or die "$!";
 
    while (my($url, $context_ref) = each %meta_docs) {
        my $digest = $digests{$url};
        unless (defined $digest) {
            my $attr = get_attr($context_ref);
            my $label_ref = $nb->predict(attributes => $attr);
            my ($pos_value, $neg_value) = @$label_ref{'+', '-'};
            next if $pos_value + $neg_value == 0;
            my $prob = $pos_value / ($pos_value + $neg_value);
            $package->add_URL_to_frontier( $url, $prob, [ [undef], 'L' ] );
            $instances_cnt++;
            unless ($params{silent}) {
                print STDERR "Record $instances_cnt\n" unless $instances_cnt % 200;
            }
        }
    }
    untie %meta_docs;
    print STDERR "Done!\n$instances_cnt URLs reordered\n\n" unless $params{silent};
}

sub learn {
    my $package = shift;
    print STDERR "\nStart Bayes learning...\n\n" unless $params{silent};

    tie my %learn_dumps,   'MLDBM',   $params{dump_db_file}   or die "$!";
    tie my %learn_digests, 'DB_File', $params{digest_db_file} or die "$!";
    tie my %meta_docs,     'MLDBM',   $params{metas_db_file}     or die "$!";

    my $instances_cnt = 0;
    while (my($url, $context_ref) = each %meta_docs) {
        my $digest = $learn_digests{$url};
        if (defined $digest) {
            my $info = $learn_dumps{$digest};
            my $label = $info->{label};
            next if $label ne '+' and $label ne '-';
            my $attr = get_attr($context_ref);
            $nb->add_instance(attributes => $attr, label => $label);
            $instances_cnt++;
            unless ($params{silent}) {
                print STDERR "Record $instances_cnt\n" unless $instances_cnt % 200;
            }
        }
    }
    $nb->train;
    $nb->do_purge();
    untie %learn_dumps;
    untie %learn_digests;
    untie %meta_docs;
    print STDERR "Done! $instances_cnt instances were trained\n\n" unless $params{silent};
    Storable::store($nb->{model}, $params{NB_file});
}

sub restore_apprentice {
    my $package = shift;
    print STDERR "apprentice restored!\n" unless $params{silent};
}

sub get_attr {
    my $context_ref = shift;
    my $attr_ref = {};
    for my $entry (@$context_ref) {
        my ($offset, $text) = @$entry;
        if (defined $text) {
            #$attr_ref->{"$_:$offset"}++ for grep { $_ and not exists $StopWords{$_} } split /[^a-z]+/, lc $text;
            $attr_ref->{"$_:$offset"}++ for lc($text) =~ /[a-z]+/g;
        }
    }
    return $attr_ref;
}

sub parse_postback {
    my $package = shift;
    my ($content, $url, $base, $label, undef, $path, undef) = @_;
    return 0 if $label eq '-';
    my $links_count = 0;
    my $root = HTML::TreeBuilder->new_from_content($content);
    my $index = 0;
    $root->objectify_text();
    do_leaf_numbering_recursive($root, \$index);
    $root->attr_encoded(1);
    my @anchors = $root->find_by_tag_name('a');
    foreach my $anchor (@anchors) {
        my $new_url = &get_new_url($anchor, $base);
        next unless defined $new_url;

        my $features_ref = [];
        collect_features($anchor, $root, $features_ref, $index, $window_width);

        my $priority;
        my $attr = get_attr($features_ref);
        my $label_ref = $nb->predict(attributes => $attr);
        my ($pos_value, $neg_value) = @$label_ref{'+', '-'};
        next if $pos_value + $neg_value == 0;
        $priority = $pos_value / ($pos_value + $neg_value);
        next if $priority < 0.3;
        $package->add_URL_to_frontier($new_url, $priority, [[$url], $path.'L']);
        $links_count++;
    }
    $root->delete;
    return $links_count;
}

1;
